Output(){
    echo -e "${output}"
}

disk_status=$GOOD
C_disk(){
    local i disk_name warn_value output
# 下面的for i语句遍历传入的参数，例如：E(){for i do echo $i; date done}; E a 3
    output="$(for i
    do
        disk_name=$(echo $i |awk -F',' '{print $1}')
        warn_value=$(echo $i |awk -F',' '{print $2}')
        df --output=used,avail,target "${disk_name}" |tail -n1 |awk -v v="${warn_value}" -v warn="${awk_warn}" '
        { x=$1/($1+$2);
        if(x>v) {
            print warn": " $3 " is used " x*100 "% > threshold:" v*100 "%"} 
        else
            print "GOOD: " $3 " is used " x*100 "% < threshold:" v*100 "%"
        }'
    done)"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${disk_status} == $GOOD ]]; then
        MailToAny "${mail_to}" "${hostName}: [告警]磁盘空间使用率" "$(Output |Change_to_html)" &   # MailToAny 不要在后台执行,因为${output}是局部变量,可能导致邮件内容不完整
        disk_status=$BAD
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${disk_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]磁盘空间使用率" "$(Output |Change_to_html)" &
        disk_status=$GOOD
        fi
}

cpu_status=$GOOD
cpu_warn_time=0         # 告警次数
cpu_top10=''
C_CPU(){
    local output warn_time warn_value
    warn_time=$(echo $1 | awk -F, '{print $1}'); [[ ${warn_time} -lt 1 ]] && warn_time=1
    warn_value=$(echo $1 | awk -F, '{print $2}')
    output="$(sar 1 1 |tail -n 1 | awk -v v="${warn_value}" -v warn="${awk_warn}" '{x=100-$NF; if(x>v) {print warn": CPU usage is " x "%."} else {print "GOOD: CPU usage is "x "%"}}')"
    Output
    if [[ $(echo "${output}" |grep -c WARN) == 0 ]]; then
        cpu_warn_time=0; cpu_top10=''
    else
        ((cpu_warn_time++))
        cpu_top10="${cpu_top10}\n${output}\n$(Eval_cont 'ps aux O-C |head')"
    fi
    if [[ ${cpu_warn_time} -ge ${warn_time} && ${cpu_status} == $GOOD ]]; then
        output="${cpu_top10}"
        MailToAny "${mail_to}" "${hostName}: [告警]CPU使用率连续${warn_time}次超过${warn_value}%" "$(Output |Change_to_html)" &
        cpu_status=$BAD
    elif [[ ${cpu_warn_time} -lt ${warn_time} && ${cpu_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]CPU使用率低于${warn_value}%" "$(Output |Change_to_html)" &
        cpu_status=$GOOD
    fi
}

swap_status=$GOOD
C_swap(){
    local output 
    output="$(free |tail -n1 |awk -v v="$1" -v warn="${awk_warn}" '
    {if($3>v) 
        {print warn": swap used: " $3/$2*100 "%, " $3 " kB > thresold:" v "kB"}
    else
        {print "GOOD: swap used: " $3/$2*100 "%, " $3 " kB < thresold:" v "kB"}
    }')"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${swap_status} == $GOOD ]]; then
        output="${output}
$(Eval_cont 'free -h'; Eval_cont 'top -b -n1 |head'; Eval_cont 'ps aux O-r |head')"
        MailToAny "${mail_to}" "${hostName}: [告警]SWAP空间使用率超过 $1 KB" "$(Output |Change_to_html)" &
        swap_status=$BAD
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${swap_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]SWAP空间使用率" "$(Output |Change_to_html)" &
        swap_status=$GOOD
    fi
}

port_status=$GOOD
C_port(){
    local i output cmd
    output="$(for i
    do
# “ :|telnet $host $port” 命令测试$host主机$port端口的连通性
        cmd="telnet 127.0.0.1 $i"
        : | eval ${cmd} | head -n 2 | tail -n 1 |
        awk -v v="${cmd}" -v warn="${awk_warn}" '{if(/Connected/){print "GOOD: " v} else print warn": " v}'
    done)"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${port_status} == $GOOD ]]; then
        MailToAny "${mail_to}" "${hostName}: [告警]端口telnet不通" "$(Output |Change_to_html)" &
        port_status=$BAD
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${port_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]端口telnet通过" "$(Output |Change_to_html)" &
        port_status=$GOOD
    fi
}
# telnet不管成功还是失败,都会输出stdout和stderr,返回码都是1
# 若telnet成功,第二行stdout返回"Connected to ..."
# 若telnet失败,有一行stdout返回"Trying ...",有一行stderr返回"telnet: connect to address ..."

nspeed_status=$GOOD
nspeed_warn_time=0
nspeed_fastest=''
C_nspeed(){
    local i if_name warn_value output warn_time=${checkNetSpeedWarn}
    [[ ${warn_time} == "" ]] && warn_time=1
    output="$(for i
    do
        if_name=$(echo ${i} |awk -F',' '{print $1}')
        warn_value=$(echo ${i} |awk -F',' '{print $2}')
        sar -n DEV 1 1 |grep "Average.*${if_name}" |awk -v n=${if_name} -v v="${warn_value}" -v warn="${awk_warn}" '
        {
        if ($5>v || $6>v) {     # $5:下载速度; $6:上传速度
            print warn": " n ": net speed > threshold:" v "kB/s. receive: " $5 "kB/s, send: " $6 "kB/s"}
        else
            print "GOOD: " n ": receive:" $5 "kB/s, send: " $6 "kB/s. Threshold is: " v "kB/s."
        }'
    done)"
    Output
    if [[ $(echo "${output}" |grep -c WARN) == 0 ]]; then
        nspeed_warn_time=0; nspeed_fastest=''
    else
        ((nspeed_warn_time++))
        nspeed_fastest="${nspeed_fastest}\n${output}
$(for i in $(echo "${output}"|awk -F: '/WARN/{print $2}')
do Eval_cont "iftop -t -s 2 -nNPB -o 2s -i ${i}"; done)"
    fi
    if [[ ${nspeed_warn_time} -ge ${warn_time} && ${nspeed_status} == $GOOD ]]; then
        output="${nspeed_fastest}"
        MailToAny "${mail_to}" "${hostName}: [告警]网速异常" "$(Output |Change_to_html)" &
        nspeed_status=$BAD
    elif [[ ${nspeed_warn_time} -lt ${warn_time} && ${nspeed_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]网速正常" "$(Output |Change_to_html)" &
        nspeed_status=$GOOD
        fi
}

rhost_status=$GOOD
C_rhost(){
    local i rhost rport cmd output
    output="$(for i
    do
        rhost=$(echo ${i} |awk -F',' '{print $1}')
        rport=$(echo ${i} |awk -F',' '{print $2}')
        cmd="telnet ${rhost} ${rport}"
        : | eval ${cmd} | head -n 2 | tail -n 1|
        awk -v v="${cmd}" -v warn="${awk_warn}" '{if(/Connected/){print "GOOD: " v} else print warn": " v}'
    done)"
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 && ${rhost_status} == $GOOD ]]; then
        MailToAny "${mail_to}" "${hostName}: [告警]远程主机telnet不通" "$(Output |Change_to_html)" &
        rhost_status=$BAD
    elif [[ $(echo "${output}" |grep -c WARN) == 0 && ${rhost_status} == $BAD ]]; then
        MailToAny "${mail_to}" "${hostName}: [恢复]远程主机telnet通过" "$(Output |Change_to_html)" &
        rhost_status=$GOOD
    fi
}

messagePreSize=0                   # 上次循环时,日志的大小
C_message(){
    local keyLines=0 messageLogSize=0 keyWord output
    messageLogSize=`wc -c $1 |awk '{print $1}'`
    if (( messagePreSize < messageLogSize && c != 1 )); then                                # 当 messagePreSize > messageLogSize 时,日志可能被rotate了,此时也忽略其告警
        keyLines="$(tail -c"+${messagePreSize}" "$1" |egrep -c "$2")"                       # keyLines 表示:含有报错关键字的行共有多少行
    fi                                                                                      # 当c==1时,messageLogSize 可能很大,故忽略smon第1次循环时所产生的告警
    if [[ ${keyLines} != 0 ]]; then                                         
      keyWord="$(tail -c"+${messagePreSize}" "$1" |egrep -o "$2" |head -n 1)"
      output="$(echo -e "${awk_warn}: 发现 '${keyWord}' 等报错,请检查${messageLog}日志")"
    else
      output="$(echo "GOOD: ${messageLog}日志没发现重要的报错信息.")"
    fi
    Output
    if [[ $(echo "${output}" |grep -c WARN) != 0 ]]; then
        output="${output}\n$(tail -c +${messagePreSize} ${1})"
        MailToAny "${mail_to}" "${hostName}: [告警]message日志有'${keyWord}'报错" "$(Output |Change_to_html)" &
    fi
    messagePreSize=${messageLogSize}
}
